{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "from __future__ import print_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_examples(N, batch_size):\n",
    "    A = np.random.binomial(n=1, p=0.5, size=(batch_size, N))\n",
    "    B = np.random.binomial(n=1, p=0.5, size=(batch_size, N,))\n",
    "\n",
    "    X = np.zeros((batch_size, 2 *N,), dtype=np.float32)\n",
    "    X[:,:N], X[:,N:] = A, B\n",
    "\n",
    "    Y = (A ^ B).astype(np.float32)\n",
    "    return X,Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "class Layer(object):\n",
    "    def __init__(self, input_sizes, output_size):\n",
    "        \"\"\"Cretes a neural network layer.\"\"\"\n",
    "        if type(input_sizes) != list:\n",
    "            input_sizes = [input_sizes]\n",
    "        \n",
    "        self.input_sizes = input_sizes\n",
    "        self.output_size = output_size\n",
    "                \n",
    "        self.Ws = []\n",
    "        for input_size in input_sizes:\n",
    "            tensor_W = tf.random_uniform((input_size, output_size),\n",
    "                                         -1.0 / math.sqrt(input_size),\n",
    "                                         1.0 / math.sqrt(input_size))\n",
    "            self.Ws.append(tf.Variable(tensor_W))\n",
    "\n",
    "        tensor_b = tf.zeros((output_size,))\n",
    "        self.b = tf.Variable(tensor_b)\n",
    "            \n",
    "    def __call__(self, xs):\n",
    "        if type(xs) != list:\n",
    "            xs = [xs]\n",
    "        assert len(xs) == len(self.Ws), \\\n",
    "                \"Expected %d input vectors, got %d\" % (len(self.Ws), len(x))\n",
    "        return sum([tf.matmul(x, W) for x, W in zip(xs, self.Ws)]) + self.b\n",
    "\n",
    "        \n",
    "class MLP(object):\n",
    "    def __init__(self, input_sizes, hiddens, nonlinearities):\n",
    "        self.input_sizes = input_sizes\n",
    "        self.hiddens = hiddens\n",
    "        self.input_nonlinearity, self.layer_nonlinearities = nonlinearities[0], nonlinearities[1:]\n",
    "\n",
    "        assert len(hiddens) == len(nonlinearities), \\\n",
    "                \"Number of hiddens must be equal to number of nonlinearities\"\n",
    "        \n",
    "        self.input_layer = Layer(input_sizes, hiddens[0])\n",
    "        self.layers = [Layer(h_from, h_to) for h_from, h_to in zip(hiddens[:-1], hiddens[1:])]\n",
    "\n",
    "    def __call__(self, xs):\n",
    "        if type(xs) != list:\n",
    "            xs = [xs]\n",
    "        hidden = self.input_nonlinearity(self.input_layer(xs))\n",
    "        for layer, nonlinearity in zip(self.layers, self.layer_nonlinearities):\n",
    "            hidden = nonlinearity(layer(hidden))\n",
    "        return hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception AssertionError: AssertionError() in <bound method InteractiveSession.__del__ of <tensorflow.python.client.session.InteractiveSession object at 0x7fe60050d990>> ignored\n"
     ]
    }
   ],
   "source": [
    "tf.ops.reset_default_graph()\n",
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "N = 5\n",
    "# we add a single hidden layer of size 12\n",
    "# otherwise code is similar to above\n",
    "HIDDEN_SIZE = 12\n",
    "\n",
    "x = tf.placeholder(tf.float32, (None, 2 * N), name=\"x\")\n",
    "y_golden = tf.placeholder(tf.float32, (None, N), name=\"y\")\n",
    "\n",
    "mlp = MLP(2 * N, [HIDDEN_SIZE, N], [tf.tanh, tf.sigmoid])\n",
    "y = mlp(x)\n",
    "\n",
    "cost = tf.reduce_mean(tf.square(y - y_golden))\n",
    "\n",
    "optimizer = tf.train.AdagradOptimizer(learning_rate=0.3)\n",
    "train_op = optimizer.minimize(cost)\n",
    "sess.run(tf.initialize_all_variables())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.241206\n",
      "0.246315\n",
      "0.193208\n",
      "0.107224\n",
      "0.100235\n",
      "0.0613462\n",
      "0.0480775\n",
      "0.0498072\n",
      "0.0403215\n",
      "0.0474323\n"
     ]
    }
   ],
   "source": [
    "for t in range(5000):\n",
    "    example_x, example_y = create_examples(N, 10)\n",
    "    cost_t, _ = sess.run([cost, train_op], {x: example_x, y_golden: example_y})\n",
    "    if t % 500 == 0: \n",
    "        print(cost_t.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy over 1000 examples: 98 %\n"
     ]
    }
   ],
   "source": [
    "N_EXAMPLES = 1000\n",
    "example_x, example_y = create_examples(N, N_EXAMPLES)\n",
    "is_correct = tf.less_equal(tf.abs(y - y_golden), tf.constant(0.5))\n",
    "accuracy = tf.reduce_mean(tf.cast(is_correct, \"float\"))\n",
    "\n",
    "acc_result = sess.run(accuracy, {x: example_x, y_golden: example_y})\n",
    "print(\"Accuracy over %d examples: %.0f %%\" % (N_EXAMPLES, 100.0 * acc_result))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
